/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package vagrants;

import java.io.*;
import java.net.*;
import java.sql.Date;
import java.sql.Time;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author vagrants
 */
public class PttCrawler {

    private static BufferedWriter log;
    private static BufferedWriter old_log;
    private static BufferedWriter info;
    private static String dirName;

    public static void main(String[] argv){

        /* get the 看板 name from URL and create the directory */
        String[] tmpStrs = argv[0].split("/");
        dirName = tmpStrs[tmpStrs.length-2];
        new File(dirName).mkdir();  // create the directory
        System.setProperty("user.dir", System.getProperty("user.dir").concat("/"+dirName) ) ;
        System.out.println("Change working directory to :" +  System.getProperty("user.dir") );


        /* open log file & info file */
        try {
            log = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new File(dirName+"/log"))));
            info = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new File(dirName+"/info"))));
        } catch (FileNotFoundException ex) {
            Logger.getLogger(PttCrawler.class.getName()).log(Level.SEVERE, null, ex);
        }

        /* write start up infomation in log */
        try {
            log.append("Board:" + dirName);
            log.newLine();
            log.append("Start Time:" + new Date(System.currentTimeMillis()) + " " + new Time(System.currentTimeMillis()));
            log.newLine();
            log.newLine();
            log.append("==");
            log.newLine();
        } catch (IOException ex) {
            Logger.getLogger(PttCrawler.class.getName()).log(Level.SEVERE, null, ex);
        }


        /* 開始一頁一頁的抓 article URL */
        Integer pageIndex;
        String line;
        String[] words;
        String articleID;
        String nowURL =  "";
        String nextURL = argv[0];
        String baseURL;

        words = argv[0].split("/");
        pageIndex = Integer.parseInt(words[words.length-1].replace("index","").replace(".html", ""));
        baseURL = argv[0].replace(words[words.length-1], "");

        try {
            while (nowURL.equals(nextURL) == false) {

                nowURL = baseURL.concat("index"+pageIndex+".html");
                pageIndex++;
                if(pageIndex > 1500){
                    break;
                }
                URL url = new URL(nowURL);
                URLConnection urlc = null;
                urlc = url.openConnection();
                urlc.setRequestProperty("User-agent",
                        "Mozilla/5.0 (Windows; U; Windows NT 6.0; zh-TW; rv:1.9.1.2) " +
                        "Gecko/20090729 Firefox/3.5.2 GTB5 (.NET CLR 3.5.30729)");

                BufferedReader readURL = new BufferedReader(new InputStreamReader(urlc.getInputStream(), "big5"));
                
                while ((line = readURL.readLine()) != null) {
                    try{
                        if (line.contains("下一頁")) {
                            words = line.split("\"");
                            nextURL = new String("http://www.ptt.cc".concat(words[1]));
                        } else if (line.contains("<td width=\"10\">")) {
                            line = line.replace("<td width=\"10\">", "");
                            line = line.replace("</td>", "");
                            articleID = line.toString();
                            readURL.readLine();
                            readURL.readLine();
                            line = readURL.readLine();
                            words = line.split("\"");
                            crawlArticle(articleID, "http://www.ptt.cc".concat(words[3]));
                        }
                    }catch(NullPointerException e){
                        System.out.println();
                        e.printStackTrace();
                    }
                }
                // System.out.println("nowURL : " + nowURL);
                // System.out.println("nextURL: " + nextURL);
            }
        } catch (IOException iOException) {
            iOException.printStackTrace();
        }
        System.out.println();


        try {
            log.append("==");
            log.newLine();
            log.newLine();
            log.append("End Time:" + new Date(System.currentTimeMillis()) + " " + new Time(System.currentTimeMillis()));
            log.newLine();
            log.close();
        } catch (IOException ex) {
            Logger.getLogger(PttCrawler.class.getName()).log(Level.SEVERE, null, ex);
        }

        System.out.println("Finished!");
    }

    private static void crawlArticle(String articleID, String strURL){

        try {
            Thread.sleep(10, 0);
            System.out.printf("\rBoard :"+dirName+", article ID:" + articleID);

            if(strURL.contains("deleted.html")){
                log.append(articleID + ":deleted:" + strURL);
                log.newLine();
                return;
            }
            int lineNum;
            String buffer;
            BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new File(dirName + "/" + articleID + ".txt"))));

            URL url = new URL(strURL);
            URLConnection urlc = url.openConnection();
            urlc.setRequestProperty("User-agent",
                    "Mozilla/5.0 (Windows; U; Windows NT 6.0; zh-TW; rv:1.9.1.2) " +
                    "Gecko/20090729 Firefox/3.5.2 GTB5 (.NET CLR 3.5.30729)");
            BufferedReader readURL = new BufferedReader(new InputStreamReader(urlc.getInputStream(), "big5"));

            lineNum = 0;
            while ((buffer = readURL.readLine()) != null) {
                if (buffer.contains("<div style=\"border: #999 1px solid; padding: 10px; background-color: #eee;\"><pre>")) {
                    if (buffer.contains("</pre></div>")) {
                        log.append(articleID + ":null:" + strURL);
                        log.newLine();
                        new File(dirName + "/" + articleID + ".txt").delete();
                        lineNum = -1;
                        break;
                    }
                    bw.append(buffer.replace("<div style=\"border: #999 1px solid; padding: 10px; background-color: #eee;\"><pre>", ""));
                    bw.newLine();
                    lineNum++;
                    while ((buffer = readURL.readLine()) != null) {
                        if (buffer.contains("</pre></div>")) {
                            break;
                        } else {
                            bw.append(buffer);
                            bw.newLine();
                            lineNum++;
                        }
                    }
                    if (buffer == null) {
                        log.append(articleID + ":error0:" + strURL);
                        log.newLine();
                        new File(dirName + "/" + articleID + ".txt").delete();
                        System.out.println();
                        throw new Exception("Unknown error no.0");
                    }
                    log.append(articleID + ":success," + lineNum + ":" + strURL);
                    log.newLine();
                    break;
                }
            }
            if(lineNum==0){
                log.append(articleID + ":error1:" + strURL);
                new File(dirName + "/" + articleID + ".txt").delete();
                log.newLine();
            }

            bw.close();
            readURL.close();
            
        } catch (Exception e) {
            e.printStackTrace();
        }


    }

}
